{
 "cells": [
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {
    "id": "ur8xi4C7S06n"
   },
   "outputs": [],
   "source": [
    "# Tracking Parameters and Metrics for Vertex AI Custom Training Jobs\n",
    "\n",
    "## Learning objectives\n",
    "\n",
    "In this notebook, you learn how to:\n",
    "\n",
    "1. Track training parameters and prediction metrics for a custom training job.\n",
    "2. Extract and perform analysis for all parameters and metrics within an experiment."    
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "tvgnzT1CKxrO"
   },
   "source": [
    "## Overview\n",
    "\n",
    "This notebook demonstrates how to track metrics and parameters for Vertex AI custom training jobs, and how to perform detailed analysis using this data.\n",
    "\n",
    "### Dataset\n",
    "\n",
    "This example uses the Abalone Dataset. For more information about this dataset please visit: https://archive.ics.uci.edu/ml/datasets/abalone \n",
    "\n",
    "Each learning objective will correspond to a __#TODO__ in the [student lab notebook](../labs/sdk_metric_parameter_tracking_for_custom_jobs.ipynb) -- try to complete that notebook first before reviewing this solution notebook."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "i7EUnXsZhAGF"
   },
   "source": [
    "### Install additional packages\n",
    "\n",
    "Install additional package dependencies not installed in your notebook environment."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "IaYsrh0Tc17L"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "# The Google Cloud Notebook product has specific requirements\n",
    "IS_GOOGLE_CLOUD_NOTEBOOK = os.path.exists(\"/opt/deeplearning/metadata/env_version\")\n",
    "\n",
    "# Google Cloud Notebook requires dependencies to be installed with '--user'\n",
    "USER_FLAG = \"\"\n",
    "if IS_GOOGLE_CLOUD_NOTEBOOK:\n",
    "    USER_FLAG = \"--user\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "qblyW_dcyOQA",
    "outputId": "20e310c7-f0a1-4c5c-9a5b-ca69713afae4"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: tensorflow in /opt/conda/lib/python3.7/site-packages (2.6.4)\n",
      "Collecting tensorflow\n",
      "  Downloading tensorflow-2.9.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (511.7 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m511.7/511.7 MB\u001b[0m \u001b[31m1.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hCollecting tensorflow-estimator<2.10.0,>=2.9.0rc0\n",
      "  Downloading tensorflow_estimator-2.9.0-py2.py3-none-any.whl (438 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m438.7/438.7 kB\u001b[0m \u001b[31m40.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting absl-py>=1.0.0\n",
      "  Downloading absl_py-1.0.0-py3-none-any.whl (126 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m126.7/126.7 kB\u001b[0m \u001b[31m18.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: keras-preprocessing>=1.1.1 in /opt/conda/lib/python3.7/site-packages (from tensorflow) (1.1.2)\n",
      "Collecting protobuf<3.20,>=3.9.2\n",
      "  Downloading protobuf-3.19.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.1 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.1/1.1 MB\u001b[0m \u001b[31m53.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: typing-extensions>=3.6.6 in /opt/conda/lib/python3.7/site-packages (from tensorflow) (3.10.0.2)\n",
      "Collecting tensorflow-io-gcs-filesystem>=0.23.1\n",
      "  Downloading tensorflow_io_gcs_filesystem-0.26.0-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (2.4 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.4/2.4 MB\u001b[0m \u001b[31m63.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: grpcio<2.0,>=1.24.3 in /opt/conda/lib/python3.7/site-packages (from tensorflow) (1.46.3)\n",
      "Requirement already satisfied: google-pasta>=0.1.1 in /opt/conda/lib/python3.7/site-packages (from tensorflow) (0.2.0)\n",
      "Requirement already satisfied: six>=1.12.0 in /opt/conda/lib/python3.7/site-packages (from tensorflow) (1.15.0)\n",
      "Collecting numpy>=1.20\n",
      "  Downloading numpy-1.21.6-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (15.7 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m15.7/15.7 MB\u001b[0m \u001b[31m16.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: gast<=0.4.0,>=0.2.1 in /opt/conda/lib/python3.7/site-packages (from tensorflow) (0.4.0)\n",
      "Requirement already satisfied: packaging in /opt/conda/lib/python3.7/site-packages (from tensorflow) (21.3)\n",
      "Requirement already satisfied: termcolor>=1.1.0 in /opt/conda/lib/python3.7/site-packages (from tensorflow) (1.1.0)\n",
      "Requirement already satisfied: flatbuffers<2,>=1.12 in /opt/conda/lib/python3.7/site-packages (from tensorflow) (1.12)\n",
      "Collecting libclang>=13.0.0\n",
      "  Downloading libclang-14.0.1-py2.py3-none-manylinux1_x86_64.whl (14.5 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m14.5/14.5 MB\u001b[0m \u001b[31m13.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hCollecting tensorboard<2.10,>=2.9\n",
      "  Downloading tensorboard-2.9.0-py3-none-any.whl (5.8 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.8/5.8 MB\u001b[0m \u001b[31m7.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m0:00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: opt-einsum>=2.3.2 in /opt/conda/lib/python3.7/site-packages (from tensorflow) (3.3.0)\n",
      "Requirement already satisfied: setuptools in /opt/conda/lib/python3.7/site-packages (from tensorflow) (59.8.0)\n",
      "Requirement already satisfied: wrapt>=1.11.0 in /opt/conda/lib/python3.7/site-packages (from tensorflow) (1.12.1)\n",
      "Collecting keras<2.10.0,>=2.9.0rc0\n",
      "  Downloading keras-2.9.0-py2.py3-none-any.whl (1.6 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.6/1.6 MB\u001b[0m \u001b[31m19.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: astunparse>=1.6.0 in /opt/conda/lib/python3.7/site-packages (from tensorflow) (1.6.3)\n",
      "Requirement already satisfied: h5py>=2.9.0 in /opt/conda/lib/python3.7/site-packages (from tensorflow) (3.1.0)\n",
      "Requirement already satisfied: wheel<1.0,>=0.23.0 in /opt/conda/lib/python3.7/site-packages (from astunparse>=1.6.0->tensorflow) (0.37.1)\n",
      "Requirement already satisfied: cached-property in /opt/conda/lib/python3.7/site-packages (from h5py>=2.9.0->tensorflow) (1.5.2)\n",
      "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /opt/conda/lib/python3.7/site-packages (from tensorboard<2.10,>=2.9->tensorflow) (1.8.1)\n",
      "Requirement already satisfied: markdown>=2.6.8 in /opt/conda/lib/python3.7/site-packages (from tensorboard<2.10,>=2.9->tensorflow) (3.3.7)\n",
      "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /opt/conda/lib/python3.7/site-packages (from tensorboard<2.10,>=2.9->tensorflow) (0.4.6)\n",
      "Requirement already satisfied: werkzeug>=1.0.1 in /opt/conda/lib/python3.7/site-packages (from tensorboard<2.10,>=2.9->tensorflow) (2.1.2)\n",
      "Requirement already satisfied: google-auth<3,>=1.6.3 in /opt/conda/lib/python3.7/site-packages (from tensorboard<2.10,>=2.9->tensorflow) (1.35.0)\n",
      "Requirement already satisfied: requests<3,>=2.21.0 in /opt/conda/lib/python3.7/site-packages (from tensorboard<2.10,>=2.9->tensorflow) (2.27.1)\n",
      "Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /opt/conda/lib/python3.7/site-packages (from tensorboard<2.10,>=2.9->tensorflow) (0.6.1)\n",
      "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/lib/python3.7/site-packages (from packaging->tensorflow) (3.0.9)\n",
      "Requirement already satisfied: pyasn1-modules>=0.2.1 in /opt/conda/lib/python3.7/site-packages (from google-auth<3,>=1.6.3->tensorboard<2.10,>=2.9->tensorflow) (0.2.7)\n",
      "Requirement already satisfied: rsa<5,>=3.1.4 in /opt/conda/lib/python3.7/site-packages (from google-auth<3,>=1.6.3->tensorboard<2.10,>=2.9->tensorflow) (4.8)\n",
      "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /opt/conda/lib/python3.7/site-packages (from google-auth<3,>=1.6.3->tensorboard<2.10,>=2.9->tensorflow) (4.2.4)\n",
      "Requirement already satisfied: requests-oauthlib>=0.7.0 in /opt/conda/lib/python3.7/site-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.10,>=2.9->tensorflow) (1.3.1)\n",
      "Requirement already satisfied: importlib-metadata>=4.4 in /opt/conda/lib/python3.7/site-packages (from markdown>=2.6.8->tensorboard<2.10,>=2.9->tensorflow) (4.11.4)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.7/site-packages (from requests<3,>=2.21.0->tensorboard<2.10,>=2.9->tensorflow) (3.3)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/conda/lib/python3.7/site-packages (from requests<3,>=2.21.0->tensorboard<2.10,>=2.9->tensorflow) (1.26.9)\n",
      "Requirement already satisfied: charset-normalizer~=2.0.0 in /opt/conda/lib/python3.7/site-packages (from requests<3,>=2.21.0->tensorboard<2.10,>=2.9->tensorflow) (2.0.12)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.7/site-packages (from requests<3,>=2.21.0->tensorboard<2.10,>=2.9->tensorflow) (2022.5.18.1)\n",
      "Requirement already satisfied: zipp>=0.5 in /opt/conda/lib/python3.7/site-packages (from importlib-metadata>=4.4->markdown>=2.6.8->tensorboard<2.10,>=2.9->tensorflow) (3.8.0)\n",
      "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /opt/conda/lib/python3.7/site-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard<2.10,>=2.9->tensorflow) (0.4.8)\n",
      "Requirement already satisfied: oauthlib>=3.0.0 in /opt/conda/lib/python3.7/site-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.10,>=2.9->tensorflow) (3.2.0)\n",
      "Installing collected packages: libclang, keras, tensorflow-io-gcs-filesystem, tensorflow-estimator, protobuf, numpy, absl-py, tensorboard, tensorflow\n",
      "\u001b[33m  WARNING: The scripts f2py, f2py3 and f2py3.7 are installed in '/home/jupyter/.local/bin' which is not on PATH.\n",
      "  Consider adding this directory to PATH or, if you prefer to suppress this warning, use --no-warn-script-location.\u001b[0m\u001b[33m\n",
      "\u001b[0m\u001b[33m  WARNING: The script tensorboard is installed in '/home/jupyter/.local/bin' which is not on PATH.\n",
      "  Consider adding this directory to PATH or, if you prefer to suppress this warning, use --no-warn-script-location.\u001b[0m\u001b[33m\n",
      "\u001b[0m\u001b[33m  WARNING: The scripts estimator_ckpt_converter, import_pb_to_tensorboard, saved_model_cli, tensorboard, tf_upgrade_v2, tflite_convert, toco and toco_from_protos are installed in '/home/jupyter/.local/bin' which is not on PATH.\n",
      "  Consider adding this directory to PATH or, if you prefer to suppress this warning, use --no-warn-script-location.\u001b[0m\u001b[33m\n",
      "\u001b[0m\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
      "tfx-bsl 1.8.0 requires google-api-python-client<2,>=1.7.11, but you have google-api-python-client 2.49.0 which is incompatible.\n",
      "tfx-bsl 1.8.0 requires pyarrow<6,>=1, but you have pyarrow 8.0.0 which is incompatible.\n",
      "tensorflow-transform 1.8.0 requires pyarrow<6,>=1, but you have pyarrow 8.0.0 which is incompatible.\n",
      "tensorflow-transform 1.8.0 requires tensorflow!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,<2.9,>=1.15.5, but you have tensorflow 2.9.1 which is incompatible.\n",
      "tensorflow-io 0.21.0 requires tensorflow<2.7.0,>=2.6.0, but you have tensorflow 2.9.1 which is incompatible.\n",
      "tensorflow-io 0.21.0 requires tensorflow-io-gcs-filesystem==0.21.0, but you have tensorflow-io-gcs-filesystem 0.26.0 which is incompatible.\n",
      "google-cloud-recommendations-ai 0.2.0 requires google-api-core[grpc]<2.0.0dev,>=1.22.2, but you have google-api-core 2.8.0 which is incompatible.\n",
      "apache-beam 2.39.0 requires dill<0.3.2,>=0.3.1.1, but you have dill 0.3.5.1 which is incompatible.\n",
      "apache-beam 2.39.0 requires httplib2<0.20.0,>=0.8, but you have httplib2 0.20.4 which is incompatible.\n",
      "apache-beam 2.39.0 requires pyarrow<8.0.0,>=0.15.1, but you have pyarrow 8.0.0 which is incompatible.\u001b[0m\u001b[31m\n",
      "\u001b[0mSuccessfully installed absl-py-1.0.0 keras-2.9.0 libclang-14.0.1 numpy-1.21.6 protobuf-3.19.4 tensorboard-2.9.0 tensorflow-2.9.1 tensorflow-estimator-2.9.0 tensorflow-io-gcs-filesystem-0.26.0\n",
      "Requirement already satisfied: google-cloud-aiplatform in /opt/conda/lib/python3.7/site-packages (1.13.0)\n",
      "Collecting google-cloud-aiplatform\n",
      "  Downloading google_cloud_aiplatform-1.13.1-py2.py3-none-any.whl (1.8 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.8/1.8 MB\u001b[0m \u001b[31m9.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: proto-plus>=1.15.0 in /opt/conda/lib/python3.7/site-packages (from google-cloud-aiplatform) (1.20.4)\n",
      "Requirement already satisfied: packaging>=14.3 in /opt/conda/lib/python3.7/site-packages (from google-cloud-aiplatform) (21.3)\n",
      "Requirement already satisfied: google-cloud-bigquery<3.0.0dev,>=1.15.0 in /opt/conda/lib/python3.7/site-packages (from google-cloud-aiplatform) (2.34.3)\n",
      "Requirement already satisfied: google-cloud-resource-manager<3.0.0dev,>=1.3.3 in /opt/conda/lib/python3.7/site-packages (from google-cloud-aiplatform) (1.5.0)\n",
      "Requirement already satisfied: google-cloud-storage<3.0.0dev,>=1.32.0 in /opt/conda/lib/python3.7/site-packages (from google-cloud-aiplatform) (2.3.0)\n",
      "Requirement already satisfied: google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0,<3.0.0dev,>=1.31.5 in /opt/conda/lib/python3.7/site-packages (from google-cloud-aiplatform) (2.8.0)\n",
      "Requirement already satisfied: google-auth<3.0dev,>=1.25.0 in /opt/conda/lib/python3.7/site-packages (from google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0,<3.0.0dev,>=1.31.5->google-cloud-aiplatform) (1.35.0)\n",
      "Requirement already satisfied: googleapis-common-protos<2.0dev,>=1.52.0 in /opt/conda/lib/python3.7/site-packages (from google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0,<3.0.0dev,>=1.31.5->google-cloud-aiplatform) (1.56.1)\n",
      "Requirement already satisfied: requests<3.0.0dev,>=2.18.0 in /opt/conda/lib/python3.7/site-packages (from google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0,<3.0.0dev,>=1.31.5->google-cloud-aiplatform) (2.27.1)\n",
      "Requirement already satisfied: protobuf>=3.12.0 in ./.local/lib/python3.7/site-packages (from google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0,<3.0.0dev,>=1.31.5->google-cloud-aiplatform) (3.19.4)\n",
      "Requirement already satisfied: grpcio<2.0dev,>=1.33.2 in /opt/conda/lib/python3.7/site-packages (from google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0,<3.0.0dev,>=1.31.5->google-cloud-aiplatform) (1.46.3)\n",
      "Requirement already satisfied: grpcio-status<2.0dev,>=1.33.2 in /opt/conda/lib/python3.7/site-packages (from google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0,<3.0.0dev,>=1.31.5->google-cloud-aiplatform) (1.46.3)\n",
      "Requirement already satisfied: google-cloud-core<3.0.0dev,>=1.4.1 in /opt/conda/lib/python3.7/site-packages (from google-cloud-bigquery<3.0.0dev,>=1.15.0->google-cloud-aiplatform) (2.3.0)\n",
      "Requirement already satisfied: python-dateutil<3.0dev,>=2.7.2 in /opt/conda/lib/python3.7/site-packages (from google-cloud-bigquery<3.0.0dev,>=1.15.0->google-cloud-aiplatform) (2.8.2)\n",
      "Requirement already satisfied: google-resumable-media<3.0dev,>=0.6.0 in /opt/conda/lib/python3.7/site-packages (from google-cloud-bigquery<3.0.0dev,>=1.15.0->google-cloud-aiplatform) (2.3.3)\n",
      "Requirement already satisfied: grpc-google-iam-v1<1.0.0dev,>=0.12.3 in /opt/conda/lib/python3.7/site-packages (from google-cloud-resource-manager<3.0.0dev,>=1.3.3->google-cloud-aiplatform) (0.12.4)\n",
      "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /opt/conda/lib/python3.7/site-packages (from packaging>=14.3->google-cloud-aiplatform) (3.0.9)\n",
      "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /opt/conda/lib/python3.7/site-packages (from google-auth<3.0dev,>=1.25.0->google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0,<3.0.0dev,>=1.31.5->google-cloud-aiplatform) (4.2.4)\n",
      "Requirement already satisfied: six>=1.9.0 in /opt/conda/lib/python3.7/site-packages (from google-auth<3.0dev,>=1.25.0->google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0,<3.0.0dev,>=1.31.5->google-cloud-aiplatform) (1.15.0)\n",
      "Requirement already satisfied: rsa<5,>=3.1.4 in /opt/conda/lib/python3.7/site-packages (from google-auth<3.0dev,>=1.25.0->google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0,<3.0.0dev,>=1.31.5->google-cloud-aiplatform) (4.8)\n",
      "Requirement already satisfied: pyasn1-modules>=0.2.1 in /opt/conda/lib/python3.7/site-packages (from google-auth<3.0dev,>=1.25.0->google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0,<3.0.0dev,>=1.31.5->google-cloud-aiplatform) (0.2.7)\n",
      "Requirement already satisfied: setuptools>=40.3.0 in /opt/conda/lib/python3.7/site-packages (from google-auth<3.0dev,>=1.25.0->google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0,<3.0.0dev,>=1.31.5->google-cloud-aiplatform) (59.8.0)\n",
      "Requirement already satisfied: google-crc32c<2.0dev,>=1.0 in /opt/conda/lib/python3.7/site-packages (from google-resumable-media<3.0dev,>=0.6.0->google-cloud-bigquery<3.0.0dev,>=1.15.0->google-cloud-aiplatform) (1.1.2)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.7/site-packages (from requests<3.0.0dev,>=2.18.0->google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0,<3.0.0dev,>=1.31.5->google-cloud-aiplatform) (2022.5.18.1)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.7/site-packages (from requests<3.0.0dev,>=2.18.0->google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0,<3.0.0dev,>=1.31.5->google-cloud-aiplatform) (3.3)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/conda/lib/python3.7/site-packages (from requests<3.0.0dev,>=2.18.0->google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0,<3.0.0dev,>=1.31.5->google-cloud-aiplatform) (1.26.9)\n",
      "Requirement already satisfied: charset-normalizer~=2.0.0 in /opt/conda/lib/python3.7/site-packages (from requests<3.0.0dev,>=2.18.0->google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0,<3.0.0dev,>=1.31.5->google-cloud-aiplatform) (2.0.12)\n",
      "Requirement already satisfied: cffi>=1.0.0 in /opt/conda/lib/python3.7/site-packages (from google-crc32c<2.0dev,>=1.0->google-resumable-media<3.0dev,>=0.6.0->google-cloud-bigquery<3.0.0dev,>=1.15.0->google-cloud-aiplatform) (1.15.0)\n",
      "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /opt/conda/lib/python3.7/site-packages (from pyasn1-modules>=0.2.1->google-auth<3.0dev,>=1.25.0->google-api-core[grpc]!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0,<3.0.0dev,>=1.31.5->google-cloud-aiplatform) (0.4.8)\n",
      "Requirement already satisfied: pycparser in /opt/conda/lib/python3.7/site-packages (from cffi>=1.0.0->google-crc32c<2.0dev,>=1.0->google-resumable-media<3.0dev,>=0.6.0->google-cloud-bigquery<3.0.0dev,>=1.15.0->google-cloud-aiplatform) (2.21)\n",
      "Installing collected packages: google-cloud-aiplatform\n",
      "\u001b[33m  WARNING: The script tb-gcp-uploader is installed in '/home/jupyter/.local/bin' which is not on PATH.\n",
      "  Consider adding this directory to PATH or, if you prefer to suppress this warning, use --no-warn-script-location.\u001b[0m\u001b[33m\n",
      "\u001b[0mSuccessfully installed google-cloud-aiplatform-1.13.1\n",
      "Requirement already satisfied: scikit-learn in /opt/conda/lib/python3.7/site-packages (1.0.2)\n",
      "Requirement already satisfied: scipy>=1.1.0 in /opt/conda/lib/python3.7/site-packages (from scikit-learn) (1.7.3)\n",
      "Requirement already satisfied: numpy>=1.14.6 in ./.local/lib/python3.7/site-packages (from scikit-learn) (1.21.6)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in /opt/conda/lib/python3.7/site-packages (from scikit-learn) (3.1.0)\n",
      "Requirement already satisfied: joblib>=0.11 in /opt/conda/lib/python3.7/site-packages (from scikit-learn) (1.1.0)\n"
     ]
    }
   ],
   "source": [
    "# Install additional packages\n",
    "! pip3 install -U tensorflow $USER_FLAG\n",
    "! python3 -m pip install {USER_FLAG} google-cloud-aiplatform --upgrade\n",
    "! pip3 install scikit-learn {USER_FLAG}\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "hhq5zEbGg0XX"
   },
   "source": [
    "**Please ignore the incompatibility errors.**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "hhq5zEbGg0XX"
   },
   "source": [
    "### Restart the kernel\n",
    "\n",
    "After you install the additional packages, you need to restart the notebook kernel so it can find the packages."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "id": "EzrelQZ22IZj"
   },
   "outputs": [],
   "source": [
    "# Automatically restart kernel after installs\n",
    "import os\n",
    "\n",
    "if not os.getenv(\"IS_TESTING\"):\n",
    "    # Automatically restart kernel after installs\n",
    "    import IPython\n",
    "\n",
    "    app = IPython.Application.instance()\n",
    "    app.kernel.do_shutdown(True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "BF1j6f9HApxa"
   },
   "source": [
    "### Set up your Google Cloud project\n",
    "\n",
    "**The following steps are required, regardless of your notebook environment.**\n",
    "\n",
    "1. [Enable the Vertex AI API and Compute Engine API](https://console.cloud.google.com/flows/enableapi?apiid=aiplatform.googleapis.com,compute_component).\n",
    "\n",
    "1. If you are running this notebook locally, you will need to install the [Cloud SDK](https://cloud.google.com/sdk).\n",
    "\n",
    "1. Enter your project ID in the cell below. Then run the cell to make sure the\n",
    "Cloud SDK uses the right project for all the commands in this notebook.\n",
    "\n",
    "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "WReHDGG5g0XY"
   },
   "source": [
    "#### Set your project ID\n",
    "\n",
    "**If you don't know your project ID**, you may be able to get your project ID using `gcloud`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "oM1iC_MfAts1",
    "outputId": "b287a8e1-d88b-4ec3-81f0-01e29ef0806c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Project ID:  qwiklabs-gcp-03-aaf99941e8b2\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "\n",
    "PROJECT_ID = \"qwiklabs-gcp-03-aaf99941e8b2\"  # Replace your project ID here  \n",
    "\n",
    "# Get your Google Cloud project ID from gcloud\n",
    "if not os.getenv(\"IS_TESTING\"):\n",
    "    shell_output = !gcloud config list --format 'value(core.project)' 2>/dev/null\n",
    "    PROJECT_ID = shell_output[0]\n",
    "    print(\"Project ID: \", PROJECT_ID)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "qJYoRfYng0XZ"
   },
   "source": [
    "Otherwise, set your project ID here."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "riG_qUokg0XZ"
   },
   "outputs": [],
   "source": [
    "if PROJECT_ID == \"\" or PROJECT_ID is None:\n",
    "    PROJECT_ID = \"qwiklabs-gcp-03-aaf99941e8b2\"  # Replace your project ID here"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "XsnuGoJM9mUw"
   },
   "source": [
    "Set gcloud config to your project ID."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "TL9QIaVd9hvm",
    "outputId": "e90b31f3-c316-45b6-e05e-ce2d51adab2f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Updated property [core/project].\n"
     ]
    }
   ],
   "source": [
    "!gcloud config set project $PROJECT_ID"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "06571eb4063b"
   },
   "source": [
    "#### Timestamp\n",
    "\n",
    "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append it onto the name of resources you create in this tutorial."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "id": "697568e92bd6"
   },
   "outputs": [],
   "source": [
    "# Import necessary library and define Timestamp\n",
    "from datetime import datetime\n",
    "\n",
    "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "zgPO1eR3CYjk"
   },
   "source": [
    "### Create a Cloud Storage bucket\n",
    "\n",
    "**The following steps are required, regardless of your notebook environment.**\n",
    "\n",
    "\n",
    "When you submit a training job using the Cloud SDK, you upload a Python package\n",
    "containing your training code to a Cloud Storage bucket. Vertex AI runs\n",
    "the code from this package. In this tutorial, Vertex AI also saves the\n",
    "trained model that results from your job in the same bucket. Using this model artifact, you can then\n",
    "create Vertex AI model and endpoint resources in order to serve\n",
    "online predictions.\n",
    "\n",
    "Set the name of your Cloud Storage bucket below. It must be unique across all\n",
    "Cloud Storage buckets.\n",
    "\n",
    "You may also change the `REGION` variable, which is used for operations\n",
    "throughout the rest of this notebook. Make sure to [choose a region where Vertex AI services are\n",
    "available](https://cloud.google.com/vertex-ai/docs/general/locations#available_regions). You may\n",
    "not use a Multi-Regional Storage bucket for training with Vertex AI."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "id": "MzGDU7TWdts_"
   },
   "outputs": [],
   "source": [
    "BUCKET_URI = \"gs://qwiklabs-gcp-03-aaf99941e8b2\"  # Replace your bucket name here\n",
    "REGION = \"us-central1\"  # @param {type:\"string\"}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "id": "cf221059d072"
   },
   "outputs": [],
   "source": [
    "if BUCKET_URI == \"\" or BUCKET_URI is None or BUCKET_URI == \"gs://qwiklabs-gcp-03-aaf99941e8b2\":  # Replace your bucket name here\n",
    "    BUCKET_URI = \"gs://\" + PROJECT_ID + \"-aip-\" + TIMESTAMP\n",
    "\n",
    "if REGION == \"[your-region]\":\n",
    "    REGION = \"us-central1\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "-EcIXiGsCePi"
   },
   "source": [
    "**Only if your bucket doesn't already exist**: Run the following cell to create your Cloud Storage bucket."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "NIq7R4HZCfIc",
    "outputId": "c997fd12-525f-4f11-8e59-1f342e5fa7a0"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating gs://qwiklabs-gcp-03-aaf99941e8b2-aip-20220530124201/...\n"
     ]
    }
   ],
   "source": [
    "# Create your bucket\n",
    "! gcloud storage buckets create --location=$REGION $BUCKET_URI"   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ucvCsknMCims"
   },
   "source": [
    "Finally, validate access to your Cloud Storage bucket by examining its contents:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "vhOb7YnwClBb",
    "outputId": "f05d7bce-5983-4297-87ba-69423084b5d3"
   },
   "outputs": [],
   "source": [
    "! gcloud storage ls --all-versions --long $BUCKET_URI"   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "XoEqT2Y4DJmf"
   },
   "source": [
    "### Import libraries and define constants"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Y9Uo3tifg1kx"
   },
   "source": [
    "Import required libraries.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "id": "pRUOFELefqf1"
   },
   "outputs": [],
   "source": [
    "# Import required libraries\n",
    "import pandas as pd\n",
    "from google.cloud import aiplatform\n",
    "from sklearn.metrics import mean_absolute_error, mean_squared_error\n",
    "from tensorflow.python.keras.utils import data_utils"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "O8XJZB3gR8eL"
   },
   "source": [
    "## Initialize Vertex AI and set an _experiment_\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "xtXZWmYqJ1bh"
   },
   "source": [
    "Define experiment name."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "id": "JIOrI-hoJ46P"
   },
   "outputs": [],
   "source": [
    "EXPERIMENT_NAME = \"new\"  # Give your experiment a name of you choice"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "jWQLXXNVN4Lv"
   },
   "source": [
    "If EXEPERIMENT_NAME is not set, set a default one below:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "id": "Q1QInYWOKsmo"
   },
   "outputs": [],
   "source": [
    "if EXPERIMENT_NAME == \"\" or EXPERIMENT_NAME is None:\n",
    "    EXPERIMENT_NAME = \"my-experiment-\" + TIMESTAMP"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "DKIsYVjj56_X"
   },
   "source": [
    "Initialize the *client* for Vertex AI."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 499
    },
    "id": "Wrlk2B2nJ7-X",
    "outputId": "2204b72f-e3eb-417d-f0f7-18d1756d7312"
   },
   "outputs": [],
   "source": [
    "aiplatform.init(\n",
    "    project=PROJECT_ID,\n",
    "    location=REGION,\n",
    "    staging_bucket=BUCKET_URI,\n",
    "    experiment=EXPERIMENT_NAME,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "6PlilQPFeS_h"
   },
   "source": [
    "## Tracking parameters and metrics in Vertex AI custom training jobs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "9nokDKBAxwV8"
   },
   "source": [
    "This example uses the Abalone Dataset. For more information about this dataset please visit: https://archive.ics.uci.edu/ml/datasets/abalone"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "id": "V_T10yTTqcS_"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2022-05-30 12:44:32--  https://storage.googleapis.com/download.tensorflow.org/data/abalone_train.csv\n",
      "Resolving storage.googleapis.com (storage.googleapis.com)... 173.194.202.128, 74.125.135.128, 74.125.142.128, ...\n",
      "Connecting to storage.googleapis.com (storage.googleapis.com)|173.194.202.128|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 145915 (142K) [text/csv]\n",
      "Saving to: ‘abalone_train.csv’\n",
      "\n",
      "abalone_train.csv   100%[===================>] 142.50K  --.-KB/s    in 0.002s  \n",
      "\n",
      "2022-05-30 12:44:32 (84.1 MB/s) - ‘abalone_train.csv’ saved [145915/145915]\n",
      "\n",
      "Copying file://abalone_train.csv [Content-Type=text/csv]...\n",
      "/ [1 files][142.5 KiB/142.5 KiB]                                                \n",
      "Operation completed over 1 objects/142.5 KiB.                                    \n"
     ]
    }
   ],
   "source": [
    "Download and copy the csv file in your bucket\n",
    "!wget https://storage.googleapis.com/download.tensorflow.org/data/abalone_train.csv\n",
    "!gcloud storage cp abalone_train.csv {BUCKET_URI}/data/\n",    "\n",
    "gcs_csv_path = f\"{BUCKET_URI}/data/abalone_train.csv\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "35QVNhACqcTJ"
   },
   "source": [
    "### Create a managed tabular dataset from a CSV\n",
    "\n",
    "A Managed dataset can be used to create an AutoML model or a custom model. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "id": "4OfCqaYRqcTJ"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating TabularDataset\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:google.cloud.aiplatform.datasets.dataset:Creating TabularDataset\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Create TabularDataset backing LRO: projects/267663067341/locations/us-central1/datasets/8583399094884499456/operations/3199694904024367104\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:google.cloud.aiplatform.datasets.dataset:Create TabularDataset backing LRO: projects/267663067341/locations/us-central1/datasets/8583399094884499456/operations/3199694904024367104\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TabularDataset created. Resource name: projects/267663067341/locations/us-central1/datasets/8583399094884499456\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:google.cloud.aiplatform.datasets.dataset:TabularDataset created. Resource name: projects/267663067341/locations/us-central1/datasets/8583399094884499456\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "To use this TabularDataset in another session:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:google.cloud.aiplatform.datasets.dataset:To use this TabularDataset in another session:\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ds = aiplatform.TabularDataset('projects/267663067341/locations/us-central1/datasets/8583399094884499456')\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:google.cloud.aiplatform.datasets.dataset:ds = aiplatform.TabularDataset('projects/267663067341/locations/us-central1/datasets/8583399094884499456')\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'projects/267663067341/locations/us-central1/datasets/8583399094884499456'"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create a managed tabular dataset\n",
    "# TODO 1\n",
    "ds = aiplatform.TabularDataset.create(display_name=\"abalone\", gcs_source=[gcs_csv_path])\n",
    "\n",
    "ds.resource_name"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "VcEOYYolqcTN"
   },
   "source": [
    "### Write the training script\n",
    "\n",
    "Run the following cell to create the training script that is used in the sample custom training job."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "id": "OauJqJmJqcTO"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Writing training_script.py\n"
     ]
    }
   ],
   "source": [
    "%%writefile training_script.py\n",
    "\n",
    "import pandas as pd\n",
    "import argparse\n",
    "import os\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras import layers\n",
    "\n",
    "parser = argparse.ArgumentParser()\n",
    "parser.add_argument('--epochs', dest='epochs',\n",
    "                    default=10, type=int,\n",
    "                    help='Number of epochs.')\n",
    "parser.add_argument('--num_units', dest='num_units',\n",
    "                    default=64, type=int,\n",
    "                    help='Number of unit for first layer.')\n",
    "args = parser.parse_args()\n",
    "# uncomment and bump up replica_count for distributed training\n",
    "# strategy = tf.distribute.experimental.MultiWorkerMirroredStrategy()\n",
    "# tf.distribute.experimental_set_strategy(strategy)\n",
    "\n",
    "col_names = [\"Length\", \"Diameter\", \"Height\", \"Whole weight\", \"Shucked weight\", \"Viscera weight\", \"Shell weight\", \"Age\"]\n",
    "target = \"Age\"\n",
    "\n",
    "def aip_data_to_dataframe(wild_card_path):\n",
    "    return pd.concat([pd.read_csv(fp.numpy().decode(), names=col_names)\n",
    "                      for fp in tf.data.Dataset.list_files([wild_card_path])])\n",
    "\n",
    "def get_features_and_labels(df):\n",
    "    return df.drop(target, axis=1).values, df[target].values\n",
    "\n",
    "def data_prep(wild_card_path):\n",
    "    return get_features_and_labels(aip_data_to_dataframe(wild_card_path))\n",
    "\n",
    "\n",
    "model = tf.keras.Sequential([layers.Dense(args.num_units), layers.Dense(1)])\n",
    "model.compile(loss='mse', optimizer='adam')\n",
    "\n",
    "model.fit(*data_prep(os.environ[\"AIP_TRAINING_DATA_URI\"]),\n",
    "          epochs=args.epochs ,\n",
    "          validation_data=data_prep(os.environ[\"AIP_VALIDATION_DATA_URI\"]))\n",
    "print(model.evaluate(*data_prep(os.environ[\"AIP_TEST_DATA_URI\"])))\n",
    "\n",
    "# save as Vertex AI Managed model\n",
    "tf.saved_model.save(model, os.environ[\"AIP_MODEL_DIR\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Yp2clkOJSDhR"
   },
   "source": [
    "### Launch a custom training job and track its training parameters on Vertex AI ML Metadata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "id": "btb6d48lqcTT"
   },
   "outputs": [],
   "source": [
    "# Define the training parameters\n",
    "job = aiplatform.CustomTrainingJob(\n",
    "    display_name=\"train-abalone-dist-1-replica\",\n",
    "    script_path=\"training_script.py\",\n",
    "    container_uri=\"us-docker.pkg.dev/vertex-ai/training/tf-cpu.2-8:latest\",\n",
    "    requirements=[\"gcsfs==0.7.1\"],\n",
    "    model_serving_container_image_uri=\"us-docker.pkg.dev/vertex-ai/prediction/tf2-cpu.2-8:latest\",\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "k_QorXXztzPH"
   },
   "source": [
    "Start a new experiment run to track training parameters and start the training job. Note that this operation will take around 10 mins."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "id": "oVTORjQpJ7-Y"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training script copied to:\n",
      "gs://qwiklabs-gcp-03-aaf99941e8b2-aip-20220530124201/aiplatform-2022-05-30-12:45:44.320-aiplatform_custom_trainer_script-0.1.tar.gz.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:google.cloud.aiplatform.utils.source_utils:Training script copied to:\n",
      "gs://qwiklabs-gcp-03-aaf99941e8b2-aip-20220530124201/aiplatform-2022-05-30-12:45:44.320-aiplatform_custom_trainer_script-0.1.tar.gz.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Output directory:\n",
      "gs://qwiklabs-gcp-03-aaf99941e8b2-aip-20220530124201/aiplatform-custom-training-2022-05-30-12:45:44.434 \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:google.cloud.aiplatform.training_jobs:Training Output directory:\n",
      "gs://qwiklabs-gcp-03-aaf99941e8b2-aip-20220530124201/aiplatform-custom-training-2022-05-30-12:45:44.434 \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No dataset split provided. The service will use a default split.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:google.cloud.aiplatform.training_jobs:No dataset split provided. The service will use a default split.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "View Training:\n",
      "https://console.cloud.google.com/ai/platform/locations/us-central1/training/697129404672770048?project=267663067341\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:google.cloud.aiplatform.training_jobs:View Training:\n",
      "https://console.cloud.google.com/ai/platform/locations/us-central1/training/697129404672770048?project=267663067341\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CustomTrainingJob projects/267663067341/locations/us-central1/trainingPipelines/697129404672770048 current state:\n",
      "PipelineState.PIPELINE_STATE_PENDING\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:google.cloud.aiplatform.training_jobs:CustomTrainingJob projects/267663067341/locations/us-central1/trainingPipelines/697129404672770048 current state:\n",
      "PipelineState.PIPELINE_STATE_PENDING\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CustomTrainingJob projects/267663067341/locations/us-central1/trainingPipelines/697129404672770048 current state:\n",
      "PipelineState.PIPELINE_STATE_RUNNING\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:google.cloud.aiplatform.training_jobs:CustomTrainingJob projects/267663067341/locations/us-central1/trainingPipelines/697129404672770048 current state:\n",
      "PipelineState.PIPELINE_STATE_RUNNING\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CustomTrainingJob projects/267663067341/locations/us-central1/trainingPipelines/697129404672770048 current state:\n",
      "PipelineState.PIPELINE_STATE_RUNNING\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:google.cloud.aiplatform.training_jobs:CustomTrainingJob projects/267663067341/locations/us-central1/trainingPipelines/697129404672770048 current state:\n",
      "PipelineState.PIPELINE_STATE_RUNNING\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CustomTrainingJob projects/267663067341/locations/us-central1/trainingPipelines/697129404672770048 current state:\n",
      "PipelineState.PIPELINE_STATE_RUNNING\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:google.cloud.aiplatform.training_jobs:CustomTrainingJob projects/267663067341/locations/us-central1/trainingPipelines/697129404672770048 current state:\n",
      "PipelineState.PIPELINE_STATE_RUNNING\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "View backing custom job:\n",
      "https://console.cloud.google.com/ai/platform/locations/us-central1/training/8439204192215629824?project=267663067341\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:google.cloud.aiplatform.training_jobs:View backing custom job:\n",
      "https://console.cloud.google.com/ai/platform/locations/us-central1/training/8439204192215629824?project=267663067341\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CustomTrainingJob projects/267663067341/locations/us-central1/trainingPipelines/697129404672770048 current state:\n",
      "PipelineState.PIPELINE_STATE_RUNNING\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:google.cloud.aiplatform.training_jobs:CustomTrainingJob projects/267663067341/locations/us-central1/trainingPipelines/697129404672770048 current state:\n",
      "PipelineState.PIPELINE_STATE_RUNNING\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CustomTrainingJob projects/267663067341/locations/us-central1/trainingPipelines/697129404672770048 current state:\n",
      "PipelineState.PIPELINE_STATE_RUNNING\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:google.cloud.aiplatform.training_jobs:CustomTrainingJob projects/267663067341/locations/us-central1/trainingPipelines/697129404672770048 current state:\n",
      "PipelineState.PIPELINE_STATE_RUNNING\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CustomTrainingJob run completed. Resource name: projects/267663067341/locations/us-central1/trainingPipelines/697129404672770048\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:google.cloud.aiplatform.training_jobs:CustomTrainingJob run completed. Resource name: projects/267663067341/locations/us-central1/trainingPipelines/697129404672770048\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model available at projects/267663067341/locations/us-central1/models/5212833005797638144\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:google.cloud.aiplatform.training_jobs:Model available at projects/267663067341/locations/us-central1/models/5212833005797638144\n"
     ]
    }
   ],
   "source": [
    "aiplatform.start_run(\"custom-training-run-1\")  # Change this to your desired run name\n",
    "parameters = {\"epochs\": 10, \"num_units\": 64}\n",
    "aiplatform.log_params(parameters)\n",
    "\n",
    "# Launch the training job\n",
    "# TODO 2\n",
    "model = job.run(\n",
    "    ds,\n",
    "    replica_count=1,\n",
    "    model_display_name=\"abalone-model\",\n",
    "    args=[f\"--epochs={parameters['epochs']}\", f\"--num_units={parameters['num_units']}\"],\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "5vhDsMJNqcTW"
   },
   "source": [
    "### Deploy Model and calculate prediction metrics"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "O-uCOL3Naap4"
   },
   "source": [
    "Deploy model to Google Cloud. This operation will take 10-20 mins."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "id": "Y9GH72wWqcTX"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Creating Endpoint\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:google.cloud.aiplatform.models:Creating Endpoint\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Create Endpoint backing LRO: projects/267663067341/locations/us-central1/endpoints/2380319528832729088/operations/61811863653974016\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:google.cloud.aiplatform.models:Create Endpoint backing LRO: projects/267663067341/locations/us-central1/endpoints/2380319528832729088/operations/61811863653974016\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Endpoint created. Resource name: projects/267663067341/locations/us-central1/endpoints/2380319528832729088\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:google.cloud.aiplatform.models:Endpoint created. Resource name: projects/267663067341/locations/us-central1/endpoints/2380319528832729088\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "To use this Endpoint in another session:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:google.cloud.aiplatform.models:To use this Endpoint in another session:\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "endpoint = aiplatform.Endpoint('projects/267663067341/locations/us-central1/endpoints/2380319528832729088')\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:google.cloud.aiplatform.models:endpoint = aiplatform.Endpoint('projects/267663067341/locations/us-central1/endpoints/2380319528832729088')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Deploying model to Endpoint : projects/267663067341/locations/us-central1/endpoints/2380319528832729088\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:google.cloud.aiplatform.models:Deploying model to Endpoint : projects/267663067341/locations/us-central1/endpoints/2380319528832729088\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Deploy Endpoint model backing LRO: projects/267663067341/locations/us-central1/endpoints/2380319528832729088/operations/1027833983724945408\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:google.cloud.aiplatform.models:Deploy Endpoint model backing LRO: projects/267663067341/locations/us-central1/endpoints/2380319528832729088/operations/1027833983724945408\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Endpoint model deployed. Resource name: projects/267663067341/locations/us-central1/endpoints/2380319528832729088\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:google.cloud.aiplatform.models:Endpoint model deployed. Resource name: projects/267663067341/locations/us-central1/endpoints/2380319528832729088\n"
     ]
    }
   ],
   "source": [
    "# Deploy the model\n",
    "# TODO 3\n",
    "endpoint = model.deploy(machine_type=\"n1-standard-4\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "JY-5skFhasWs"
   },
   "source": [
    "Once model is deployed, perform online prediction using the `abalone_test` dataset and calculate prediction metrics."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "saw50bqwa-dR"
   },
   "source": [
    "Prepare the prediction dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "id": "ABZQmqsWISQv"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading data from https://storage.googleapis.com/download.tensorflow.org/data/abalone_test.csv\n",
      "40960/37298 [================================] - 0s 0us/step\n"
     ]
    }
   ],
   "source": [
    "def read_data(uri):\n",
    "    dataset_path = data_utils.get_file(\"abalone_test.data\", uri)\n",
    "    col_names = [\n",
    "        \"Length\",\n",
    "        \"Diameter\",\n",
    "        \"Height\",\n",
    "        \"Whole weight\",\n",
    "        \"Shucked weight\",\n",
    "        \"Viscera weight\",\n",
    "        \"Shell weight\",\n",
    "        \"Age\",\n",
    "    ]\n",
    "    dataset = pd.read_csv(\n",
    "        dataset_path,\n",
    "        names=col_names,\n",
    "        na_values=\"?\",\n",
    "        comment=\"\\t\",\n",
    "        sep=\",\",\n",
    "        skipinitialspace=True,\n",
    "    )\n",
    "    return dataset\n",
    "\n",
    "\n",
    "def get_features_and_labels(df):\n",
    "    target = \"Age\"\n",
    "    return df.drop(target, axis=1).values, df[target].values\n",
    "\n",
    "\n",
    "test_dataset, test_labels = get_features_and_labels(\n",
    "    read_data(\n",
    "        \"https://storage.googleapis.com/download.tensorflow.org/data/abalone_test.csv\"\n",
    "    )\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "_HphZ38obJeB"
   },
   "source": [
    "Perform online prediction."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "id": "eXD-OvsrKmCt"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Prediction(predictions=[[11.6409063], [12.4500084], [8.29393387], [11.338728], [9.49171448], [11.6065626], [8.01098919], [11.6627703], [12.7768526], [14.533371], [10.40242], [8.77202], [11.2494688], [11.1994419], [11.1550875], [12.113369], [9.99136353], [11.0012226], [9.33863], [13.831027], [11.9634705], [8.16823483], [8.28015137], [11.8544569], [10.0856323], [10.3430099], [8.25908], [8.07181358], [8.6134], [10.9741917], [6.81412458], [12.5263824], [8.75480461], [7.29967499], [9.01774788], [9.99683475], [12.4618635], [8.12143421], [10.8554497], [10.2606096], [10.0540285], [13.3028402], [7.62779045], [6.79086113], [10.3587093], [7.08481121], [10.5466175], [8.71292591], [10.0715351], [8.38328648], [8.73528576], [10.7041225], [12.1339569], [9.35072136], [11.7982054], [14.4456005], [11.9373531], [11.5827904], [11.4634895], [9.36929321], [12.2990704], [10.943984], [12.1526127], [9.39513111], [11.103878], [8.58725452], [11.638814], [6.27024698], [11.7717037], [12.5781231], [9.22437763], [9.96546078], [9.96971321], [11.1706963], [10.9819918], [9.76925278], [11.0085735], [11.3836441], [10.6172371], [11.4222355], [5.64062691], [10.553751], [13.0663204], [9.38279533], [10.1953678], [8.63195229], [8.16534519], [10.6624928], [11.6315022], [7.0363574], [10.9442368], [11.4874363], [5.51654053], [10.8529587], [14.4359], [7.86314344], [6.72594118], [9.18144417], [10.3312578], [10.249115], [8.12998199], [7.95897436], [11.0812063], [8.27920532], [7.10126209], [7.74954176], [6.64678431], [9.39654827], [8.61367798], [9.52682686], [11.5111675], [7.80128527], [9.87465668], [10.3159733], [8.36497116], [11.6024971], [8.49816799], [10.8901558], [6.00905], [8.80128288], [11.6479187], [8.79641342], [12.2356224], [11.5552692], [10.1999397], [10.3948202], [14.6354609], [7.70099497], [12.0115328], [8.52399158], [9.97729683], [10.9418926], [7.16012335], [7.41653728], [7.32016754], [10.4602528], [7.81726503], [10.974966], [9.10255146], [9.69795895], [7.71325827], [5.48800707], [9.61999512], [7.38872576], [10.2052135], [11.3361797], [11.8184481], [11.6087208], [11.4153671], [10.2047863], [8.79839516], [8.52340508], [7.32520342], [9.49822903], [11.1208553], [8.54162598], [9.88449287], [11.0817451], [12.3864565], [7.21285486], [7.25776052], [11.5855675], [12.9321079], [9.92162514], [9.43742943], [5.19743776], [9.57037354], [13.2558174], [8.48231506], [7.47348547], [8.30984], [8.99649811], [11.4792852], [9.86909866], [14.114109], [9.3258152], [12.1418228], [6.5491128], [5.2847476], [8.69965839], [8.46738148], [7.64852095], [13.0742826], [7.85100365], [7.51083422], [9.63409233], [8.38929558], [12.9331169], [13.0376396], [9.05260468], [10.6164045], [13.1252317], [9.37162781], [8.17575455], [7.00188255], [10.0777092], [13.4249229], [7.41486406], [8.20608], [12.419178], [8.87764168], [12.6553917], [10.6202154], [8.05703354], [10.6471], [8.30507851], [7.75193834], [10.7226877], [10.8681698], [13.0148106], [6.81926775], [6.95566702], [8.53998375], [11.5034142], [11.5512047], [10.0805416], [7.88287926], [10.2450047], [11.6715031], [13.1340866], [11.3394651], [7.09081364], [12.0328846], [8.3078022], [9.83701134], [13.7374859], [8.73082447], [11.7468376], [6.71510267], [9.12092209], [9.69717216], [6.00917912], [11.3785877], [6.32208109], [12.747], [12.1714067], [8.67528725], [9.74445343], [10.4982481], [11.0656776], [12.4989061], [9.08342934], [10.9417248], [9.90470886], [11.7544842], [7.29588556], [13.3063354], [11.2352333], [13.4868116], [10.0982704], [9.82918167], [7.21349716], [9.19985199], [11.8230038], [6.94323301], [10.2613201], [12.5437737], [11.4248981], [12.823843], [11.7486782], [10.1132708], [5.24786472], [15.321537], [11.9538536], [10.1226063], [9.17489433], [10.6074209], [6.66961908], [10.4056721], [10.5530529], [11.3925495], [8.21489811], [10.5877304], [9.56541824], [11.0915937], [8.50979], [10.5969028], [11.1313477], [9.39159775], [8.51598454], [7.36100149], [11.411788], [9.34882], [8.07995605], [11.5648184], [9.28127193], [12.089757], [8.8807745], [10.0935841], [9.9401474], [12.7125244], [6.86083841], [7.17849207], [11.6913395], [7.09705305], [10.2519064], [8.98278809], [8.43517494], [8.55042648], [10.321043], [9.04671288], [11.1759319], [8.48534], [11.8352394], [12.6079903], [9.74123], [7.56206703], [11.6255789], [12.1017551], [12.0510473], [9.63322639], [12.1680822], [10.7772], [10.4901886], [11.2958584], [10.0626936], [9.31879807], [8.46724701], [9.0333519], [10.6157551], [9.36319637], [11.3018885], [10.2590322], [6.5192], [7.45523643], [12.04632], [8.98111343], [11.695467], [9.2127285], [9.99427605], [6.63702488], [12.1692429], [9.45989418], [10.0974751], [15.3263388], [8.54658127], [10.5291672], [8.81067], [12.0271492], [9.13233662], [14.8301487], [10.5171156], [11.4049549], [10.9421024], [10.0267391], [12.2651358], [13.9209156], [10.7872086], [10.4359913], [8.96704483], [6.97460127], [12.207819], [9.94033337], [10.1410961], [9.83991909], [11.7227917], [9.24898338], [8.06538868], [11.7193995], [10.6062202], [8.70868301], [8.31696129], [7.30157185], [7.06776094], [11.0554142], [9.97211933], [9.81086826], [9.16932487], [12.4864044], [8.37537289], [15.1236439], [6.96791315], [8.81603241], [8.72799683], [6.78659344], [12.6970272], [11.3353291], [11.3004484], [8.70714188], [10.6657543], [11.7024736], [9.85241318], [9.23450089], [8.03086662], [6.84616137], [10.573822], [11.459281], [10.4782314], [6.33672428], [13.111846], [10.1794243], [7.1473608], [11.4658337], [10.1417313], [11.526741], [13.30933], [10.712719], [8.84378242], [9.03753853], [7.57823324], [11.7480745], [8.27678299], [10.27495], [15.3893795], [10.9013605], [10.2322044], [8.48244762], [10.8324938], [9.41399097], [14.3796864], [6.44309664], [14.6098633], [8.46298313], [11.6659832], [10.3172665], [11.2361202], [10.2959261], [7.58545923], [10.1717281], [10.5570087], [11.2556782], [11.7384911], [7.14396667], [11.8346157], [6.44107771], [6.61311102], [11.5042562], [8.72333622], [10.8882627], [11.9640779], [8.0529871], [8.77625656], [13.3984203], [6.63243198], [14.6636639], [10.2897739], [11.2451849], [10.545104], [8.93007851], [6.41319799], [8.96269798], [7.43786], [7.4153471], [9.90675449], [8.96344], [8.37816525], [10.4487209], [9.98269844], [7.5619359], [13.3167], [10.1600494], [9.54284859], [10.0573339], [9.28747177], [10.68643], [9.32512665], [7.85834408], [11.3175516], [11.6108932], [9.94993591], [10.9439583], [9.76673508], [6.28919554], [12.5527821], [9.01048279], [10.5293598], [6.85526419], [12.0611467], [10.6316013], [9.86765766], [8.69333553], [11.2434855], [6.95810938], [12.0016766], [8.89791489], [9.56030083], [11.4206123], [12.0183125], [9.54114914], [6.46475792], [11.789319], [9.16521], [10.0256367], [11.9128742], [7.59778214], [9.66135597], [11.2413502], [7.4686265], [10.8266201], [10.3905029], [9.90942192], [12.3887119], [7.99272728], [7.19865847], [10.3490334], [8.70278168], [11.6948481], [12.7428217], [13.0314274], [10.5840044], [7.4307704], [12.0244846], [10.0945587], [11.3745594], [10.6245203], [7.27501583], [11.5918941], [11.1316242], [10.6969376], [7.09594631], [10.285902], [6.2159667], [9.24382], [12.4499741], [11.2488155], [9.08309937], [9.6669817], [10.0215569], [9.73712826], [5.00597334], [11.7815237], [11.186224], [7.29885], [9.4430294], [10.7344685], [7.7261138], [10.2874689], [11.3405323], [9.81174278], [9.95916557], [10.2164488], [8.71251106], [10.5838528], [7.88245821], [11.8100233], [9.94266891], [9.11565208], [10.1822529], [9.91239929], [10.6549406], [10.2875509], [12.6981754], [7.90556335], [11.89328], [9.1484766], [12.7993393], [11.3889465], [8.66545391], [7.25700712], [8.58734894], [10.6466255], [8.7752676], [14.3699236], [9.28340244], [7.08472586], [6.91889715], [11.7416229], [11.2480202], [8.07476616], [13.2546062], [9.65588474], [8.54018211], [11.8548822], [10.2484646], [13.9830475], [9.2875042], [7.32192707], [11.8710299], [12.9082489], [8.59631443], [11.3462257], [6.42396402], [12.7599163], [9.23474312], [8.22327614], [9.74340534], [10.064312], [12.2650661], [10.1714048], [9.1644], [8.70679951], [11.5623398], [7.01665831], [11.6984081], [11.670454], [7.8431716], [11.8970356], [10.2520018], [9.27325726], [7.81853962], [13.6915913], [10.4801025], [11.5724277], [7.73723745], [11.1017399], [11.3849373], [9.8432579], [7.66264629], [10.3208799], [10.0047264], [10.2351446], [8.19178], [9.63452911], [9.59336376], [10.1366329], [7.19121885], [12.352952], [9.74002838], [12.887228], [7.93140173], [8.48637867], [10.3808031], [12.2411766], [7.73703337], [8.24461365], [11.1596804], [7.18046045], [8.48374462], [11.1103878], [10.6270714], [11.6828804], [15.2590103], [12.0632553], [10.0884466], [10.2996273], [9.62692833], [8.50937843], [12.9846125], [12.0354404], [12.536438], [11.3525858], [10.3029041], [9.21931267], [9.35728645], [7.23363781], [8.23759651], [11.3278942], [10.4364443], [5.43523502], [8.31951904], [11.1675043], [14.068182], [11.9655457], [11.8512516], [10.7548075], [8.86014], [11.137351], [9.10429573], [8.60079765], [7.69010258], [8.78439522], [6.29921865], [11.6228104], [11.0465765], [8.75774384], [10.0316067], [12.03654], [6.85030603], [11.1709824], [11.9903965], [8.71390152], [10.7613], [7.39155149], [11.1403923], [6.95580101], [13.1047258], [12.6830597], [6.2560792], [11.5519886], [9.67996693], [8.47209835], [11.2261744], [8.66270447], [12.3794298], [10.3747158], [12.1412201], [9.69344902], [12.4364071], [9.43394566], [9.63906], [9.00484467], [11.6325779], [10.117548], [8.99370289], [14.1568556], [8.22506523], [11.0568409], [6.88945], [9.46739769], [11.1166191], [13.4896164], [11.4113522], [8.27496338], [7.7850728], [12.6035137], [10.8592014], [10.4925337], [8.62538147], [7.50395346], [12.4162121], [13.3722353], [9.75363541], [10.4090271], [10.0383358], [11.1422777], [11.5604916], [11.3814583], [13.3555708], [6.3706069], [11.6048174], [7.21637106], [5.42589426], [9.74350071], [11.1327553], [13.7560501], [9.89745903], [11.0674133], [11.766386], [11.8369579], [10.9581709], [8.74327374], [10.9149694], [10.6663074], [15.6479912], [9.3988018], [11.2886658], [8.26124], [12.5830231], [12.1501455], [9.32803726], [8.6666069], [8.97563362], [11.7256813], [10.6311073], [13.4952469], [9.35672], [9.707901], [10.6694717], [11.4401417], [10.7798643], [7.47313118], [12.1573095], [10.0586405], [9.27191734], [6.47630882], [13.4458656], [10.124629], [10.1195316], [10.4996786], [5.9364028], [10.7311563], [6.8705492], [8.74482], [12.5490723], [7.08100653], [10.3703766], [10.8133144], [11.7708702], [11.3138962], [8.50624943], [8.38814545], [9.35721588], [11.4713774], [13.9424562], [8.1516819], [11.2253571], [10.1197796], [10.7568445], [8.83505249], [9.85730362], [11.3544245], [6.24070311], [10.8946762], [7.00799131], [9.08911514], [11.2910652], [8.06574535], [11.9538841], [7.15557814], [8.68668938], [8.13148689], [9.22748375], [11.3418293], [8.26413345], [10.0690508], [10.4477234], [11.251049], [7.18021822], [8.51853752], [9.21061325], [8.34187126], [12.0037413], [12.1138153], [8.77697563], [11.628685], [10.1066399], [10.0384083], [9.00790787], [11.168787], [12.5478172], [9.88206863], [7.0711937], [10.4584198], [8.35082436], [10.2792673], [8.00649166], [10.0049438], [13.7647629], [11.333415], [12.2980156], [9.3295126], [7.93553972], [5.99963331], [9.87442], [9.49925518], [12.1586571], [8.16145325], [12.2965221], [9.71896172], [8.42557], [11.2249012], [12.6040649], [10.0934849], [10.4452114], [11.1846371], [8.94481087], [9.08661366], [10.441535], [14.1549625], [7.62100887], [11.6159782], [12.4606295], [12.0624151], [13.3940039], [9.62810707], [6.69409895], [11.3847551], [7.67718744], [9.72272873], [12.013217], [8.40690804], [10.8780479], [10.4819679], [13.9559793], [10.8852215], [10.9488106], [10.2371578], [11.307003], [9.67247772], [10.8972244]], deployed_model_id='1732188210584354816', explanations=None)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Perform online prediction using endpoint\n",
    "# TODO 4\n",
    "prediction = endpoint.predict(test_dataset.tolist())\n",
    "prediction"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "TDKiv_O7bNwE"
   },
   "source": [
    "Calculate and track prediction evaluation metrics."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "id": "cj0fHucbKopn"
   },
   "outputs": [],
   "source": [
    "mse = mean_squared_error(test_labels, prediction.predictions)\n",
    "mae = mean_absolute_error(test_labels, prediction.predictions)\n",
    "\n",
    "aiplatform.log_metrics({\"mse\": mse, \"mae\": mae})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "CCGmesdIbbHf"
   },
   "source": [
    "### Extract all parameters and metrics created during this experiment."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "id": "KlcEBou-Pl4Z"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>experiment_name</th>\n",
       "      <th>run_name</th>\n",
       "      <th>param.num_units</th>\n",
       "      <th>param.epochs</th>\n",
       "      <th>metric.mse</th>\n",
       "      <th>metric.mae</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>new</td>\n",
       "      <td>custom-training-run-1</td>\n",
       "      <td>64.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>7.251621</td>\n",
       "      <td>1.940373</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  experiment_name               run_name  param.num_units  param.epochs  \\\n",
       "0             new  custom-training-run-1             64.0          10.0   \n",
       "\n",
       "   metric.mse  metric.mae  \n",
       "0    7.251621    1.940373  "
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Extract all parameters and metrics of the experiment\n",
    "# TODO 5\n",
    "aiplatform.get_experiment_df()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "WTHvPMweMlP1"
   },
   "source": [
    "### View data in the Cloud Console"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "F19_5lw0MqXv"
   },
   "source": [
    "Parameters and metrics can also be viewed in the Cloud Console. \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "id": "GmN9vE9pqqzt"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Vertex AI Experiments:\n",
      "https://console.cloud.google.com/ai/platform/experiments/experiments?folder=&organizationId=&project=qwiklabs-gcp-03-aaf99941e8b2\n"
     ]
    }
   ],
   "source": [
    "print(\"Vertex AI Experiments:\")\n",
    "print(\n",
    "    f\"https://console.cloud.google.com/ai/platform/experiments/experiments?folder=&organizationId=&project={PROJECT_ID}\"\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "TpV-iwP9qw9c"
   },
   "source": [
    "## Cleaning up\n",
    "\n",
    "To clean up all Google Cloud resources used in this project, you can [delete the Google Cloud\n",
    "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
    "\n",
    "Otherwise, you can delete the individual resources you created in this tutorial:\n",
    "Training Job\n",
    "Model\n",
    "Cloud Storage Bucket\n",
    "\n",
    "* Vertex AI Dataset\n",
    "* Training Job\n",
    "* Model\n",
    "* Endpoint\n",
    "* Cloud Storage Bucket\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "id": "rwPZoZISHhaY"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Deleting TabularDataset : projects/267663067341/locations/us-central1/datasets/8583399094884499456\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:google.cloud.aiplatform.base:Deleting TabularDataset : projects/267663067341/locations/us-central1/datasets/8583399094884499456\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Delete TabularDataset  backing LRO: projects/267663067341/locations/us-central1/operations/3876360748036784128\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:google.cloud.aiplatform.base:Delete TabularDataset  backing LRO: projects/267663067341/locations/us-central1/operations/3876360748036784128\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TabularDataset deleted. . Resource name: projects/267663067341/locations/us-central1/datasets/8583399094884499456\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:google.cloud.aiplatform.base:TabularDataset deleted. . Resource name: projects/267663067341/locations/us-central1/datasets/8583399094884499456\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Deleting CustomTrainingJob : projects/267663067341/locations/us-central1/trainingPipelines/697129404672770048\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:google.cloud.aiplatform.base:Deleting CustomTrainingJob : projects/267663067341/locations/us-central1/trainingPipelines/697129404672770048\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Delete CustomTrainingJob  backing LRO: projects/267663067341/locations/us-central1/operations/2748209041380474880\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:google.cloud.aiplatform.base:Delete CustomTrainingJob  backing LRO: projects/267663067341/locations/us-central1/operations/2748209041380474880\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CustomTrainingJob deleted. . Resource name: projects/267663067341/locations/us-central1/trainingPipelines/697129404672770048\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:google.cloud.aiplatform.base:CustomTrainingJob deleted. . Resource name: projects/267663067341/locations/us-central1/trainingPipelines/697129404672770048\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Undeploying Endpoint model: projects/267663067341/locations/us-central1/endpoints/2380319528832729088\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:google.cloud.aiplatform.models:Undeploying Endpoint model: projects/267663067341/locations/us-central1/endpoints/2380319528832729088\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Undeploy Endpoint model backing LRO: projects/267663067341/locations/us-central1/endpoints/2380319528832729088/operations/1519852243015172096\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:google.cloud.aiplatform.models:Undeploy Endpoint model backing LRO: projects/267663067341/locations/us-central1/endpoints/2380319528832729088/operations/1519852243015172096\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Endpoint model undeployed. Resource name: projects/267663067341/locations/us-central1/endpoints/2380319528832729088\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:google.cloud.aiplatform.models:Endpoint model undeployed. Resource name: projects/267663067341/locations/us-central1/endpoints/2380319528832729088\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Deleting Endpoint : projects/267663067341/locations/us-central1/endpoints/2380319528832729088\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:google.cloud.aiplatform.base:Deleting Endpoint : projects/267663067341/locations/us-central1/endpoints/2380319528832729088\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Delete Endpoint  backing LRO: projects/267663067341/locations/us-central1/operations/7095308581699846144\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:google.cloud.aiplatform.base:Delete Endpoint  backing LRO: projects/267663067341/locations/us-central1/operations/7095308581699846144\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Endpoint deleted. . Resource name: projects/267663067341/locations/us-central1/endpoints/2380319528832729088\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:google.cloud.aiplatform.base:Endpoint deleted. . Resource name: projects/267663067341/locations/us-central1/endpoints/2380319528832729088\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Deleting Model : projects/267663067341/locations/us-central1/models/5212833005797638144\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:google.cloud.aiplatform.base:Deleting Model : projects/267663067341/locations/us-central1/models/5212833005797638144\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Delete Model  backing LRO: projects/267663067341/locations/us-central1/operations/4647039234270560256\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:google.cloud.aiplatform.base:Delete Model  backing LRO: projects/267663067341/locations/us-central1/operations/4647039234270560256\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model deleted. . Resource name: projects/267663067341/locations/us-central1/models/5212833005797638144\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:google.cloud.aiplatform.base:Model deleted. . Resource name: projects/267663067341/locations/us-central1/models/5212833005797638144\n"
     ]
    }
   ],
   "source": [
    "# Warning: Setting this to true will delete everything in your bucket\n",
    "delete_bucket = False\n",
    "\n",
    "# Delete dataset\n",
    "ds.delete()\n",
    "\n",
    "# Delete the training job\n",
    "job.delete()\n",
    "\n",
    "# Undeploy model from endpoint\n",
    "endpoint.undeploy_all()\n",
    "\n",
    "# Delete the endpoint\n",
    "endpoint.delete()\n",
    "\n",
    "# Delete the model\n",
    "model.delete()\n",
    "\n",
    "\n",
    "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
    "    ! gcloud storage rm --recursive $BUCKET_URI"   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "sdk-metric-parameter-tracking-for-custom-jobs.ipynb",
   "provenance": [],
   "toc_visible": true
  },
  "environment": {
   "kernel": "python3",
   "name": "tf2-gpu.2-6.m93",
   "type": "gcloud",
   "uri": "gcr.io/deeplearning-platform-release/tf2-gpu.2-6:m93"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
